import cv2
import os
import numpy as np
import seaborn as sns 
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing import image
from PIL import Image
import tensorflow as trf
from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Input, BatchNormalization
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix, classification_report
from keras.optimizers import Adam

import warnings
warnings.filterwarnings('ignore')

dataset_folder = r'C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val'
classes = ['COVID-19', 'Non-COVID', 'Normal']

Data Preprocessing

Checking for image balance

Checking for the balance of images is important because an imbalanced dataset can bias the model towards the majority class and lead to overfitting. So, let's check the balance of images.

def plot_class_distribution(data_dirs, xlabel_dict):
    for data_dir in data_dirs:
        class_counts = {}
        total_samples = 0
        class_name = os.path.basename(data_dir)
        x_labels = xlabel_dict.get(class_name, ["Images", "Lung Masks"])
        
        for subdir, xlabel in zip(["Images", "Lung Masks"], x_labels):
            subdir_path = os.path.join(data_dir, subdir)
            num_samples = len(os.listdir(subdir_path))
            class_counts[xlabel] = num_samples
            total_samples += num_samples
        
        plt.figure(figsize=(9, 6))
        plt.bar(class_counts.keys(), class_counts.values())
        plt.xlabel('Subdirectory')
        plt.ylabel('Number of Samples')
        plt.title(f'Class Distribution for {class_name}')
        plt.xticks(rotation=45, ha='right')
        plt.tight_layout()
        plt.show()

        print(f"Total number of samples for {class_name}: {total_samples}")

data_directories = [
    r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\COVID-19",
    r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Normal",
    r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Non-COVID"
]

xlabel_dict = {
    "COVID-19": ["covid Images", "Covid Lung Masks"],
    "Normal": ["nrmal Images", "Normal Lung Masks"],
    "Non-COVID": ["non-Covid Images", "Non-Covid Lung Masks"]
}

plot_class_distribution(data_directories, xlabel_dict)

Total number of samples for COVID-19: 3806

Total number of samples for Normal: 3424

Total number of samples for Non-COVID: 3604

We don't have imbalanced data (images). There's a difference of around 100 images. Therefore, there is no need to balance the images.

Image Resizing

Checking for input size.

image_path = r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Non-COVID\images\non_COVID (6143).png"  # Replace with the path to your image file
image = Image.open(image_path)
image_size = image.size
print("Image size:", image_size)

Image size: (256, 256)

Since original images are 256*256 pixels, we could consider resizing them to a commonly used input size such as (224, 224) or (299, 299)

for resizing the images either we can downsample it or crop the images.

As the lung is located in the middle of the image, we can consider cropping the image.

Image Normalization

First, let's understand what normalization is and why we need to perform it.

Normalization is like putting all our data on the same scale to make it fair and balanced. It ensures consistency, allows for fair comparison, and improves performance in data analysis. By converting measurements to a common scale, normalization helps us analyze data more effectively and draw meaningful insights from it.

Performing Resizing and Normalizing

#target size for resizing
target_size = (224, 224)

#performing MinMaxScaler
scaler = MinMaxScaler()
normalized_images = []
normalized_masks = []
labels = []

#load and resizing 
def load_and_resize_image(image_path, target_size):
    img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
    resized_img = cv2.resize(img, target_size)
    return resized_img

# Loop through each class folder
for class_index, class_name in enumerate(classes):
    image_folder = os.path.join(dataset_folder, class_name, 'images')
    mask_folder = os.path.join(dataset_folder, class_name, 'lung masks')

    for filename in os.listdir(image_folder):
        image_path = os.path.join(image_folder, filename)
        mask_path = os.path.join(mask_folder, filename)

        # Load and resize images
        original_img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
        resized_img = load_and_resize_image(image_path, target_size)
        
        # Normalize the resized image
        normalized_img = scaler.fit_transform(resized_img.astype(float).reshape(-1, 1)).reshape(resized_img.shape)

       # Load and resize masks
        mask_img = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
        resized_mask = load_and_resize_image(mask_path, target_size)

        # Normalizinge resized mask
        normalized_mask = scaler.fit_transform(resized_mask.astype(float).reshape(-1, 1)).reshape(resized_mask.shape)
        
       # Append 
        normalized_images.append(normalized_img)
        normalized_masks.append(normalized_mask)
        labels.append(class_index) 

#lists to arrays
normalized_images = np.array(normalized_images)
normalized_masks = np.array(normalized_masks)
labels = np.array(labels)

print("Img_Size:", normalized_images.shape)
print("Masks_Size:", normalized_masks.shape)

Img_Size: (5417, 224, 224)
Masks_Size: (5417, 224, 224)

ori_pixel = original_img.flatten()
norm_pixel = normalized_img.flatten()
        
# Display original, resized, and normalized images
for i in range(7):
    plt.figure(figsize=(15, 6))
    plt.subplot(1, 4, 1)
    plt.imshow(original_img, cmap='gray')
    plt.title('Original Image')

    plt.subplot(1, 4, 2)
    plt.imshow(resized_img, cmap='gray')
    plt.title('Resized Image')

    plt.subplot(1, 4, 3)
    plt.imshow(normalized_img, cmap='gray')
    plt.title('Normalized Image')

    #before and after normalization
    print("Pixel values before normalization:")
    print(ori_pixel)
#     print()
    print("\nPixel values after normalization:")
    print(norm_pixel)
    print()

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

Pixel values before normalization:
[160 121  99 ...  70  58  35]

Pixel values after normalization:
[0.61354582 0.4501992  0.3625498  ... 0.30278884 0.24302789 0.14741036]

There is no need to normalize black and white images since they already range between 0 and 1. However, let's compare the values to confirm.

ori_pixel1 = mask_img.flatten()
norm_pixel1 = normalized_mask.flatten()

#plotting for masks
for i in range(5):
    plt.figure(figsize=(15, 6))
    plt.subplot(1, 4, 1)
    plt.imshow(mask_img, cmap='gray')
    plt.title('Original Image')

    plt.subplot(1, 4, 2)
    plt.imshow(resized_mask, cmap='gray')
    plt.title('Resized Image')

    plt.subplot(1, 4, 3)
    plt.imshow(normalized_mask, cmap='gray')
    plt.title('Normalized Image')

    #before and after normalization
    print("Pixel values before normalization:")
    print(ori_pixel1)
    print("\nPixel values after normalization:")
    print(norm_pixel1)

Pixel values before normalization:
[0 0 0 ... 0 0 0]

Pixel values after normalization:
[0. 0. 0. ... 0. 0. 0.]
Pixel values before normalization:
[0 0 0 ... 0 0 0]

Pixel values after normalization:
[0. 0. 0. ... 0. 0. 0.]
Pixel values before normalization:
[0 0 0 ... 0 0 0]

Pixel values after normalization:
[0. 0. 0. ... 0. 0. 0.]
Pixel values before normalization:
[0 0 0 ... 0 0 0]

Pixel values after normalization:
[0. 0. 0. ... 0. 0. 0.]
Pixel values before normalization:
[0 0 0 ... 0 0 0]

Pixel values after normalization:
[0. 0. 0. ... 0. 0. 0.]

We can also perform noise reduction techniques for preprocesssing such as,

(1). Gaussian Blur: This method uses a special filter called a Gaussian filter to gently soften the image, making it smoother and reducing small details. It's great for getting rid of the grainy look in pictures, especially in areas with lots of fine details.

(2). Median Filtering: Instead of averaging neighboring pixel values like many filters, median filtering takes the middle value among the neighboring pixels. This is especially useful for cleaning up images with random specks or dots (like salt-and-pepper noise) while keeping the edges of objects nice and sharp.

reference: https://medium.com/@anishaswain/noise-filtering-in-digital-image-processing-d12b5266847c

Splitting the data

# Combine preprocessed(resized and normalized) images and masks 
combined_data = list(zip(normalized_images, normalized_masks,labels))

train_data, test_data = train_test_split(combined_data, test_size=0.2, random_state=42)

# Unpack the data into separate lists for images, masks, and labels
def unpack_data(data):
    if data:
        return zip(*data)
    else:
        return [], [], []

# Unpack training data
X_train, masks_train, y_train = unpack_data(train_data)
X_test, masks_test, y_test = unpack_data(test_data)

X_train = np.array(X_train)
y_train = np.array(y_train)
masks_train = np.array(masks_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
masks_test = np.array(masks_test)

y_train

array([2, 1, 2, ..., 2, 2, 0])

CNN (Convolutional neural network)

https://towardsdatascience.com/a-comprehensive-guide-to-convolutional-neural-networks-the-eli5-way-3bd2b1164a53

Pooling layer

Max pooling selects the brighter pixels from the image. It is useful when the background of the image is dark and we are interested in only the lighter pixels of the image.

Approches

Here we have four approaches, but we're going with the averaging approach.

CNN for Images

input_shape_image = (224, 224, 1)  
input_shape_mask = (224, 224, 1) 

# CNN model for image data
model_image = Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=input_shape_image),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(3, activation='softmax')  
])

model_image.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 222, 222, 32)      320       
                                                                 
 max_pooling2d (MaxPooling2D  (None, 111, 111, 32)     0         
 )                                                               
                                                                 
 conv2d_1 (Conv2D)           (None, 109, 109, 64)      18496     
                                                                 
 max_pooling2d_1 (MaxPooling  (None, 54, 54, 64)       0         
 2D)                                                             
                                                                 
 conv2d_2 (Conv2D)           (None, 52, 52, 64)        36928     
                                                                 
 flatten (Flatten)           (None, 173056)            0         
                                                                 
 dense (Dense)               (None, 64)                11075648  
                                                                 
 dense_1 (Dense)             (None, 3)                 195       
                                                                 
=================================================================
Total params: 11,131,587
Trainable params: 11,131,587
Non-trainable params: 0
_________________________________________________________________

model_image.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# Convert labels to one-hot encoded vectors
y_train_one_hot = to_categorical(y_train, num_classes=3)
y_test_one_hot = to_categorical(y_test, num_classes=3)

# Train Image Model
model_image.fit(X_train, y_train_one_hot, epochs=7, batch_size=30)

Epoch 1/7
145/145 [==============================] - 109s 738ms/step - loss: 1.0040 - accuracy: 0.5751
Epoch 2/7
145/145 [==============================] - 106s 730ms/step - loss: 0.6508 - accuracy: 0.7390
Epoch 3/7
145/145 [==============================] - 107s 735ms/step - loss: 0.5265 - accuracy: 0.7914
Epoch 4/7
145/145 [==============================] - 104s 715ms/step - loss: 0.4110 - accuracy: 0.8401
Epoch 5/7
145/145 [==============================] - 103s 707ms/step - loss: 0.2999 - accuracy: 0.8800
Epoch 6/7
145/145 [==============================] - 105s 721ms/step - loss: 0.1942 - accuracy: 0.9301
Epoch 7/7
145/145 [==============================] - 102s 704ms/step - loss: 0.1306 - accuracy: 0.9541

<keras.callbacks.History at 0x2552f9b6b80>

# Evaluate the image model on test data
image_loss, image_accuracy = model_image.evaluate(X_test, y_test_one_hot)
print("Image Model - Test Loss:", image_loss)
print("Image Model - Test Acc:", image_accuracy)

34/34 [==============================] - 5s 131ms/step - loss: 0.7415 - accuracy: 0.7906
Image Model - Test Loss: 0.7414774894714355
Image Model - Test Acc: 0.7905904054641724

CNN for Mask Images

# CNN model for mask data
model_mask = Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=input_shape_mask),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(3, activation='softmax')
])


model_mask.summary()

Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_3 (Conv2D)           (None, 222, 222, 32)      320       
                                                                 
 max_pooling2d_2 (MaxPooling  (None, 111, 111, 32)     0         
 2D)                                                             
                                                                 
 conv2d_4 (Conv2D)           (None, 109, 109, 64)      18496     
                                                                 
 max_pooling2d_3 (MaxPooling  (None, 54, 54, 64)       0         
 2D)                                                             
                                                                 
 conv2d_5 (Conv2D)           (None, 52, 52, 64)        36928     
                                                                 
 flatten_1 (Flatten)         (None, 173056)            0         
                                                                 
 dense_2 (Dense)             (None, 64)                11075648  
                                                                 
 dense_3 (Dense)             (None, 3)                 195       
                                                                 
=================================================================
Total params: 11,131,587
Trainable params: 11,131,587
Non-trainable params: 0
_________________________________________________________________

# Compile the mask model
model_mask.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# Train Mask Model
model_mask.fit(masks_train, y_train_one_hot, epochs=7, batch_size=30)

Epoch 1/7
145/145 [==============================] - 105s 720ms/step - loss: 1.0780 - accuracy: 0.4690
Epoch 2/7
145/145 [==============================] - 100s 689ms/step - loss: 0.8719 - accuracy: 0.5844
Epoch 3/7
145/145 [==============================] - 99s 684ms/step - loss: 0.7338 - accuracy: 0.6681
Epoch 4/7
145/145 [==============================] - 98s 677ms/step - loss: 0.5077 - accuracy: 0.7895
Epoch 5/7
145/145 [==============================] - 102s 704ms/step - loss: 0.2559 - accuracy: 0.9045
Epoch 6/7
145/145 [==============================] - 97s 668ms/step - loss: 0.1084 - accuracy: 0.9603
Epoch 7/7
145/145 [==============================] - 97s 669ms/step - loss: 0.0520 - accuracy: 0.9843

<keras.callbacks.History at 0x2573cbd8640>

# Evaluate the mask model on test data
mask_loss, mask_accuracy = model_image.evaluate(masks_test,y_test_one_hot)
print("Mask Model - Test Loss:", mask_loss)
print("Mak Model - Test Accuracy:", mask_accuracy)

34/34 [==============================] - 4s 119ms/step - loss: 21.7385 - accuracy: 0.4077
Mask Model - Test Loss: 21.73845863342285
Mak Model - Test Accuracy: 0.40774908661842346

here, we can also take weighted average if we want to give more weights to either image or mask.

# Combine predictions by taking avegae
combined_pred = ((model_image.predict(X_test) + model_mask.predict(masks_test))/2)

34/34 [==============================] - 4s 127ms/step
34/34 [==============================] - 4s 121ms/step

auc_score = roc_auc_score(y_test_one_hot, combined_pred)
print("Combined AUC_score:", auc_score)

Combined AUC_score: 0.8898847877182926

# Convert predictions to class
combinedlabels = np.argmax(combined_pred, axis=1)
truelabels = np.argmax(y_test_one_hot, axis=1)

# Confusion Matrix
confmatrix = confusion_matrix(truelabels, combinedlabels, labels=np.arange(len(classes)))
plt.figure(figsize=(8, 6))
sns.heatmap(confmatrix, annot=True, fmt='d', cmap='Blues', xticklabels=classes, yticklabels=classes)
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()

# classification report
class_report = classification_report(truelabels, combinedlabels, target_names=classes)
print(class_report)

              precision    recall  f1-score   support

    COVID-19       0.72      0.73      0.73       406
   Non-COVID       0.73      0.72      0.73       328
      Normal       0.72      0.71      0.72       350

    accuracy                           0.72      1084
   macro avg       0.72      0.72      0.72      1084
weighted avg       0.72      0.72      0.72      1084

DNN (Deep Neural Network)

DNN for Images

input_shape_image = (224, 224, 1)
input_shape_mask = (224, 224, 1)
# Define the DNN model for image data
model_image = Sequential([
    Flatten(input_shape=input_shape_image),
    Dense(128, activation='relu'),
    Dense(64, activation='relu'),
    Dense(3, activation='softmax')  
])

# Compile the image model
model_image.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model_image.summary()

Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 flatten_2 (Flatten)         (None, 50176)             0         
                                                                 
 dense_4 (Dense)             (None, 128)               6422656   
                                                                 
 dense_5 (Dense)             (None, 64)                8256      
                                                                 
 dense_6 (Dense)             (None, 3)                 195       
                                                                 
=================================================================
Total params: 6,431,107
Trainable params: 6,431,107
Non-trainable params: 0
_________________________________________________________________

# Train the image model
model_image.fit(X_train, y_train_one_hot, epochs=7, batch_size=30)

Epoch 1/7
145/145 [==============================] - 10s 69ms/step - loss: 3.0698 - accuracy: 0.4853
Epoch 2/7
145/145 [==============================] - 11s 72ms/step - loss: 1.0989 - accuracy: 0.5800
Epoch 3/7
145/145 [==============================] - 10s 66ms/step - loss: 0.8901 - accuracy: 0.6227
Epoch 4/7
145/145 [==============================] - 9s 64ms/step - loss: 0.9238 - accuracy: 0.6324
Epoch 5/7
145/145 [==============================] - 9s 64ms/step - loss: 0.9694 - accuracy: 0.6185
Epoch 6/7
145/145 [==============================] - 9s 63ms/step - loss: 0.8375 - accuracy: 0.6358
Epoch 7/7
145/145 [==============================] - 9s 64ms/step - loss: 0.7964 - accuracy: 0.6580

<keras.callbacks.History at 0x2573ceeb760>

# Evaluate the image model on test data
image_loss, image_accuracy = model_image.evaluate(X_test, y_test_one_hot)
print("Image Model - Test Loss:", image_loss)
print("Image Model - Test Acc:", image_accuracy)

34/34 [==============================] - 0s 5ms/step - loss: 1.0943 - accuracy: 0.5498
Image Model - Test Loss: 1.0942562818527222
Image Model - Test Acc: 0.5498154759407043

DNN for Mask

# Define the DNN model for mask data
model_mask = Sequential([
    Flatten(input_shape=input_shape_mask),
    Dense(128, activation='relu'),
    Dense(64, activation='relu'),
    Dense(3, activation='softmax')
])

model_mask.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model_mask.summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 flatten_3 (Flatten)         (None, 50176)             0         
                                                                 
 dense_7 (Dense)             (None, 128)               6422656   
                                                                 
 dense_8 (Dense)             (None, 64)                8256      
                                                                 
 dense_9 (Dense)             (None, 3)                 195       
                                                                 
=================================================================
Total params: 6,431,107
Trainable params: 6,431,107
Non-trainable params: 0
_________________________________________________________________

# Train the mask model
model_mask.fit(masks_train, y_train_one_hot, epochs=7, batch_size=30)

Epoch 1/7
145/145 [==============================] - 10s 64ms/step - loss: 1.7174 - accuracy: 0.4607
Epoch 2/7
145/145 [==============================] - 9s 64ms/step - loss: 0.9744 - accuracy: 0.5481
Epoch 3/7
145/145 [==============================] - 9s 65ms/step - loss: 0.9155 - accuracy: 0.5763
Epoch 4/7
145/145 [==============================] - 9s 64ms/step - loss: 0.8559 - accuracy: 0.6010
Epoch 5/7
145/145 [==============================] - 9s 64ms/step - loss: 0.8251 - accuracy: 0.6160
Epoch 6/7
145/145 [==============================] - 9s 65ms/step - loss: 0.7939 - accuracy: 0.6363
Epoch 7/7
145/145 [==============================] - 9s 65ms/step - loss: 0.7697 - accuracy: 0.6423

<keras.callbacks.History at 0x2573d14e640>

# Evaluate the mask model on test data
mask_loss, mask_accuracy = model_mask.evaluate(masks_test, y_test_one_hot)
print("Mask Model - Test Loss:", mask_loss)
print("Mask Model - Test Accuracy:", mask_accuracy)

34/34 [==============================] - 0s 5ms/step - loss: 0.9566 - accuracy: 0.5913
Mask Model - Test Loss: 0.9566269516944885
Mask Model - Test Accuracy: 0.5913284420967102

# Combine predictions from both models
combined_pred = ((model_image.predict(X_test) + model_mask.predict(masks_test))/2)
combined_labels = np.argmax(combined_pred, axis=1)
true_labels = np.argmax(y_test_one_hot, axis=1)

34/34 [==============================] - 0s 5ms/step
34/34 [==============================] - 0s 5ms/step

# Confusion Matrix
conf_matrix = confusion_matrix(true_labels, combined_labels)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=classes, yticklabels=classes)
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()

auc_score = roc_auc_score(y_test_one_hot, combined_pred)
print("Combined AUC_score:", auc_score)

Combined AUC_score: 0.8397445972790241

# classification report
class_report = classification_report(truelabels, combinedlabels, target_names=classes)
print(class_report)

              precision    recall  f1-score   support

    COVID-19       0.72      0.73      0.73       406
   Non-COVID       0.73      0.72      0.73       328
      Normal       0.72      0.71      0.72       350

    accuracy                           0.72      1084
   macro avg       0.72      0.72      0.72      1084
weighted avg       0.72      0.72      0.72      1084